package org.spider;


import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.RandomAccessFile;
import java.io.UnsupportedEncodingException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;



public class DictUrlSpider {
	private static final String REPORT_PATH = "dict_urls.txt";
	private static final String SOGOU_DICT_URL_BASE = "http://pinyin.sogou.com/dict/";
	private static final String CATEGORY_URL= "(list.php\\?c=[\\d]+)";
	private static final String CATEGORY_MAX_PAGE_NUMBER = "共([\\d]+)页";
	private static final String DOWNLOAD_URL = "(http://download.pinyin.sogou.com/dict/download_cell.php[^\"]+)";
	
	public static void main(String[] args) throws UnsupportedEncodingException, IOException{
		
//		Map<String, String> headers = new HashMap<String, String>();
//		
//		headers.put("referer", "http://pinyin.sogou.com/dict/");
//		
//		HttpUtil.saveResponseToFile("d:/test.secl", "http://download.pinyin.sogou.com/dict/download_cell.php?id=9287&name=%BE%A2%CE%E8%CD%C5", headers, null);
		System.out.println("[Request]: 开始获取分类信息...");
		
		String idxPageHtml = downloadPage(SOGOU_DICT_URL_BASE);
		
		//获得分类URL列表
		List<String> categoryList = parseTextGroup(idxPageHtml, CATEGORY_URL, false, 0);
		
		System.out.println("[Result]: 获取分类成功，获得[" + categoryList.size() + "]个分类信息...");
		
		Set<String> downloadUrlSet = new LinkedHashSet<String>();
		for(String categoryUrl : categoryList){
			List<String> oneCategoryDownloadUrlList = parseCategory(SOGOU_DICT_URL_BASE + categoryUrl);
			
			for(String url : oneCategoryDownloadUrlList)
				downloadUrlSet.add(url);
		}
		
		StringBuffer result = new StringBuffer();
		
		for(String url : downloadUrlSet){
			result.append(url + "\n");
		}
		
		saveToFile(new File(REPORT_PATH), result.toString(), true);
		
		System.out.println("#####爬取结束，共抓取到[" + downloadUrlSet.size() + "]个下载链接，保存在[" + REPORT_PATH + "]");
	}
	
	private static List<String> parseCategory(String categoryUrl) throws UnsupportedEncodingException, IOException {
		List<String> result = new ArrayList<String>();
		
		System.out.println("    -[Request]: 开始获取分类页数信息 - " + categoryUrl);
		
		String cateHtml = downloadPage(categoryUrl);
		
		List<String> catePageNumber = parseTextGroup(cateHtml, CATEGORY_MAX_PAGE_NUMBER, false, 0);
		
		int maxPageNumber = (catePageNumber!=null&&catePageNumber.size()>0)?Integer.parseInt(catePageNumber.get(0)):0;
		
		System.out.println("    -[Result]: 获取分类页数成功，当前分类共[" + maxPageNumber + "]页...");
		
		for(int n=1;n<=maxPageNumber;n++){
			result.addAll(parseOnePageDownloadUrlList(categoryUrl, n));
		}
		return result;
	}

	private static Collection<String> parseOnePageDownloadUrlList(String cateUrl, int pageNumber) throws UnsupportedEncodingException, IOException {
		String pageUrl = cateUrl + "&page=" + pageNumber;
		String onePageHtml = downloadPage(pageUrl);
		
		System.out.println("        -[Request]: 开始获取第[" + pageNumber + "]页的下载链接信息 - " + pageUrl);
		Set<String> result = new LinkedHashSet<String>();
		
		result.addAll(parseTextGroup(onePageHtml, DOWNLOAD_URL, false, 0));
		
		System.out.println("        -[Result]: 获得第[" + pageNumber + "]页的下载链接成功，共[" + result.size() + "]个...");
		return result;
	}

	public static String downloadPage(String urlStr) throws UnsupportedEncodingException, IOException{
		URL url = new URL(urlStr);
		
		StringBuffer result = new StringBuffer();
		
		BufferedReader br = new BufferedReader(new InputStreamReader(url.openStream(), "gbk"));
		
		String line;
		
		while((line=br.readLine())!=null){
			result.append(line + "\n");
		}
		
		br.close();
		
		return result.toString();
	}
	
	/**
	 * 用正则提取指定分组的内容
	 * @param content
	 * @param pattern
	 * @param groupNum
	 * @return
	 */
	public static List<String> parseTextGroup(String content, String pattern, boolean caseSensitive, int groupNum){
		if(groupNum < 0)
			return null;
		
		List<List<String>> result = parseTextGroups(content, pattern, caseSensitive);
		
		if(result != null && result.size() > groupNum)
			return result.get(groupNum);
		else
			return null;
	}
	
	/**
	 * 使用正则提取分组中的内容，多个分组会以多个List<String>形式展现
	 * @param content
	 * @param pattern
	 * @return
	 */
	public static List<List<String>> parseTextGroups(String content, String pattern, boolean caseSensitive){
    	if(content == null || content.length()<=0 || pattern == null || content.length()<=0)
    		return null;
    	
    	List<List<String>> result = new ArrayList<List<String>>();
    	
    	Pattern p = null;
    	
    	if(caseSensitive)
    		p = Pattern.compile(pattern);
    	else
    		p = Pattern.compile(pattern, Pattern.CASE_INSENSITIVE);
    	
    	Matcher matcher = p.matcher(content);
    	
    	while(matcher.find()){
    		
    		for(int n=1;n<=matcher.groupCount();n++){
    			if(result.size()<n){
    				result.add(new ArrayList<String>());
    			}
    			result.get(n-1).add(matcher.group(n));
    		}
    	}
    	
    	return result;
    }
	
	public static void saveToFile(File src, String content, boolean overwrite) {
		if (src == null || content == null)
			return;

		if (!src.exists())
			try {
				src.createNewFile();
			} catch (IOException e1) {
			}

		RandomAccessFile raf = null;

		try {
			raf = new RandomAccessFile(src, "rw");
		} catch (FileNotFoundException e) {
			return;
		}
		try {
			if (!overwrite)
				raf.seek(raf.length());
			else {
				raf.setLength(0);
			}

			raf.write(content.getBytes());

		} catch (IOException e) {
			e.printStackTrace();
		} finally {
			try {
				raf.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
	}
}
